{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sad_construct import *\n",
    "from sad_read_mnist import get_mnist\n",
    "(x_train,y_train),(x_test,y_test)=get_mnist()\n",
    "x_train=x_train.reshape(-1,1*28*28)\n",
    "x_test=x_test.reshape(-1,1*28*28)\n",
    "mnist_train_data = np.append(x_train, y_train, axis=1)\n",
    "mnist_test_data = np.append(x_test, y_test, axis=1)\n",
    "mnist_shape=(1,28,28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sad_network import NeuralNetwork as Network\n",
    "from sad_layer import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init para\n",
      "epoch 1:\n",
      " Loss:0.453621107 \tAccuracy: 0.9319 [0.97653, 0.97885, 0.92054, 0.91683, 0.94196, 0.87556, 0.94885, 0.92899, 0.92813, 0.89296]\n",
      "epoch 2:\n",
      " Loss:0.217636030 \tAccuracy: 0.9481 [0.98265, 0.97885, 0.95543, 0.9505, 0.92974, 0.91256, 0.96033, 0.93969, 0.92094, 0.94252]\n",
      "epoch 3:\n",
      " Loss:0.164494780 \tAccuracy: 0.9577 [0.98367, 0.9859, 0.96512, 0.95347, 0.95927, 0.92601, 0.96973, 0.94163, 0.95483, 0.93162]\n",
      "epoch 4:\n",
      " Loss:0.132624338 \tAccuracy: 0.9627 [0.98163, 0.9859, 0.96899, 0.97426, 0.97251, 0.92489, 0.96868, 0.95623, 0.94456, 0.94252]\n",
      "epoch 5:\n",
      " Loss:0.110781747 \tAccuracy: 0.9689 [0.9898, 0.98855, 0.97481, 0.96832, 0.97454, 0.94507, 0.97077, 0.9572, 0.96715, 0.94846]\n"
     ]
    }
   ],
   "source": [
    "train_and_test(\n",
    "    mynetwork=fc_mnist,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=16,\n",
    "    epoch_num=5,\n",
    "    pth=\"fc_mnist_b16.npy\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init para\n",
      "epoch 1:\n",
      " Loss:0.574346250 \tAccuracy: 0.9155 [0.98367, 0.98062, 0.87888, 0.92178, 0.94807, 0.85202, 0.92484, 0.90175, 0.87269, 0.8781]\n",
      "epoch 2:\n",
      " Loss:0.277184165 \tAccuracy: 0.9338 [0.98061, 0.98326, 0.91764, 0.92772, 0.94603, 0.90359, 0.94676, 0.91148, 0.90246, 0.9108]\n",
      "epoch 3:\n",
      " Loss:0.225806251 \tAccuracy: 0.9419 [0.98776, 0.98238, 0.93702, 0.94059, 0.94399, 0.89798, 0.94676, 0.91732, 0.93121, 0.92567]\n",
      "epoch 4:\n",
      " Loss:0.192692634 \tAccuracy: 0.947 [0.98673, 0.98502, 0.94186, 0.94554, 0.96232, 0.93274, 0.94363, 0.9358, 0.93326, 0.89792]\n",
      "epoch 5:\n",
      " Loss:0.168631632 \tAccuracy: 0.9539 [0.98673, 0.9859, 0.95349, 0.9495, 0.95112, 0.93386, 0.95407, 0.94844, 0.93943, 0.93062]\n"
     ]
    }
   ],
   "source": [
    "train_and_test(\n",
    "    mynetwork=fc_mnist,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=5,\n",
    "    half_learning_rate_time=4,\n",
    "    pth=\"fc_mnist_b32.npy\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:297.688567671 \tAccuracy: 0.9133 \t\t\n",
      "epoch 2:\n",
      " Loss:165.832065776 \tAccuracy: 0.9331 \t\t\n",
      "epoch 3:\n",
      " Loss:138.472203473 \tAccuracy: 0.9433 \t\t\n",
      "epoch 4:\n",
      " Loss:122.427719522 \tAccuracy: 0.9489 \t\t\n",
      "epoch 5:\n",
      " Loss:111.224231740 \tAccuracy: 0.9533 \t\t\n",
      "epoch 6:\n",
      " Loss:102.734137725 \tAccuracy: 0.9546 \t\t\n",
      "epoch 7:\n",
      " Loss:95.891014244 \tAccuracy: 0.9577 \t\t\n",
      "epoch 8:\n",
      " Loss:90.332496237 \tAccuracy: 0.9603 \t\t\n",
      "epoch 9:\n",
      " Loss:85.446655432 \tAccuracy: 0.9609 \t\t\n",
      "epoch 10:\n",
      " Loss:81.334377122 \tAccuracy: 0.9623 \t\t\n"
     ]
    }
   ],
   "source": [
    "fc2=Network(\n",
    "    layers=[\n",
    "        FullConnectedLayer(784,256),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(256,64),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(64,10),\n",
    "        ReLuLayer()\n",
    "    ],\n",
    "    lossfunction=MSELossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fc2,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:362.447518961 \tAccuracy: 0.8288 \t\t\n",
      "epoch 2:\n",
      " Loss:243.295257173 \tAccuracy: 0.9052 \t\t\n",
      "epoch 3:\n",
      " Loss:155.161022935 \tAccuracy: 0.94 \t\t\n",
      "epoch 4:\n",
      " Loss:128.773030184 \tAccuracy: 0.9479 \t\t\n",
      "epoch 5:\n",
      " Loss:115.116482054 \tAccuracy: 0.9505 \t\t\n",
      "epoch 6:\n",
      " Loss:105.377094059 \tAccuracy: 0.9548 \t\t\n",
      "epoch 7:\n",
      " Loss:97.674228899 \tAccuracy: 0.9585 \t\t\n",
      "epoch 8:\n",
      " Loss:91.582992178 \tAccuracy: 0.9599 \t\t\n",
      "epoch 9:\n",
      " Loss:86.387843063 \tAccuracy: 0.9606 \t\t\n",
      "epoch 10:\n",
      " Loss:81.997627789 \tAccuracy: 0.9636 \t\t\n"
     ]
    }
   ],
   "source": [
    "fc3=Network(\n",
    "    layers=[\n",
    "        FullConnectedLayer(784,256),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(256,64),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(64,10),\n",
    "        ReLuLayer()\n",
    "    ],\n",
    "    lossfunction=HuberLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fc3,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:1145.909087307 \tAccuracy: 0.911 \t\t\n",
      "epoch 2:\n",
      " Loss:541.623343128 \tAccuracy: 0.9262 \t\t\n",
      "epoch 3:\n",
      " Loss:443.714558726 \tAccuracy: 0.9393 \t\t\n",
      "epoch 4:\n",
      " Loss:378.782127634 \tAccuracy: 0.9457 \t\t\n",
      "epoch 5:\n",
      " Loss:330.762560444 \tAccuracy: 0.9494 \t\t\n",
      "epoch 6:\n",
      " Loss:293.414628118 \tAccuracy: 0.9557 \t\t\n",
      "epoch 7:\n",
      " Loss:263.507585780 \tAccuracy: 0.9587 \t\t\n",
      "epoch 8:\n",
      " Loss:238.771669255 \tAccuracy: 0.962 \t\t\n",
      "epoch 9:\n",
      " Loss:218.096297064 \tAccuracy: 0.965 \t\t\n",
      "epoch 10:\n",
      " Loss:199.902991982 \tAccuracy: 0.9658 \t\t\n"
     ]
    }
   ],
   "source": [
    "fc4=Network(\n",
    "    layers=[\n",
    "        FullConnectedLayer(784,256),\n",
    "        LeakyReLULayer(),\n",
    "        FullConnectedLayer(256,64),\n",
    "        LeakyReLULayer(),\n",
    "        FullConnectedLayer(64,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fc4,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:1052.086083162 \tAccuracy: 0.9117 \t\t\n",
      "epoch 2:\n",
      " Loss:586.399329349 \tAccuracy: 0.923 \t\t\n",
      "epoch 3:\n",
      " Loss:500.193861982 \tAccuracy: 0.931 \t\t\n",
      "epoch 4:\n",
      " Loss:445.211027698 \tAccuracy: 0.9382 \t\t\n",
      "epoch 5:\n",
      " Loss:402.946186407 \tAccuracy: 0.9424 \t\t\n",
      "epoch 6:\n",
      " Loss:368.489241202 \tAccuracy: 0.945 \t\t\n",
      "epoch 7:\n",
      " Loss:339.527028596 \tAccuracy: 0.949 \t\t\n",
      "epoch 8:\n",
      " Loss:314.000190577 \tAccuracy: 0.952 \t\t\n",
      "epoch 9:\n",
      " Loss:292.849666449 \tAccuracy: 0.9546 \t\t\n",
      "epoch 10:\n",
      " Loss:273.857626185 \tAccuracy: 0.9571 \t\t\n"
     ]
    }
   ],
   "source": [
    "fc5=Network(\n",
    "    layers=[\n",
    "        FullConnectedLayer(784,256),\n",
    "        TanhLayer(),\n",
    "        FullConnectedLayer(256,64),\n",
    "        TanhLayer(),\n",
    "        FullConnectedLayer(64,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fc5,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:970.749196110 \tAccuracy: 0.9092 \t\t\n",
      "epoch 2:\n",
      " Loss:580.305995080 \tAccuracy: 0.9231 \t\t\n",
      "epoch 3:\n",
      " Loss:511.629111099 \tAccuracy: 0.9274 \t\t\n",
      "epoch 4:\n",
      " Loss:459.582706407 \tAccuracy: 0.9352 \t\t\n",
      "epoch 5:\n",
      " Loss:412.985048834 \tAccuracy: 0.9385 \t\t\n",
      "epoch 6:\n",
      " Loss:373.338955642 \tAccuracy: 0.9452 \t\t\n",
      "epoch 7:\n",
      " Loss:339.277055585 \tAccuracy: 0.9487 \t\t\n",
      "epoch 8:\n",
      " Loss:311.116821756 \tAccuracy: 0.9536 \t\t\n",
      "epoch 9:\n",
      " Loss:287.767839247 \tAccuracy: 0.956 \t\t\n",
      "epoch 10:\n",
      " Loss:266.875821629 \tAccuracy: 0.9583 \t\t\n"
     ]
    }
   ],
   "source": [
    "fc6=Network(\n",
    "    layers=[\n",
    "        FullConnectedLayer(784,256),\n",
    "        ELULayer(),\n",
    "        FullConnectedLayer(256,64),\n",
    "        ELULayer(),\n",
    "        FullConnectedLayer(64,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fc6,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:4317.131086929 \tAccuracy: 0.1135 \t\t\n",
      "epoch 2:\n",
      " Loss:4316.920468298 \tAccuracy: 0.1135 \t\t\n",
      "epoch 3:\n",
      " loss:2.310037345 \tprocess: 20.8%\t    "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[34], line 12\u001b[0m\n\u001b[0;32m      1\u001b[0m fc7\u001b[38;5;241m=\u001b[39mNetwork(\n\u001b[0;32m      2\u001b[0m     layers\u001b[38;5;241m=\u001b[39m[\n\u001b[0;32m      3\u001b[0m         FullConnectedLayer(\u001b[38;5;241m784\u001b[39m,\u001b[38;5;241m256\u001b[39m,std\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.001\u001b[39m),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m      9\u001b[0m     lossfunction\u001b[38;5;241m=\u001b[39mSoftmaxCrossEntropyLossLayer()\n\u001b[0;32m     10\u001b[0m )\n\u001b[1;32m---> 12\u001b[0m train_and_test(\n\u001b[0;32m     13\u001b[0m     mynetwork\u001b[38;5;241m=\u001b[39mfc7,\n\u001b[0;32m     14\u001b[0m     train_data\u001b[38;5;241m=\u001b[39mmnist_train_data,\n\u001b[0;32m     15\u001b[0m     test_data\u001b[38;5;241m=\u001b[39mmnist_test_data,\n\u001b[0;32m     16\u001b[0m     data_shape\u001b[38;5;241m=\u001b[39mmnist_shape,\n\u001b[0;32m     17\u001b[0m     onehotsize\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m,\n\u001b[0;32m     18\u001b[0m     batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m32\u001b[39m,\n\u001b[0;32m     19\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m\n\u001b[0;32m     20\u001b[0m )\n",
      "Cell \u001b[1;32mIn[18], line 10\u001b[0m, in \u001b[0;36mtrain_and_test\u001b[1;34m(mynetwork, train_data, test_data, data_shape, onehotsize, batch_size, epoch_num, learning_rate, half_learning_rate_time)\u001b[0m\n\u001b[0;32m      8\u001b[0m batch_y \u001b[38;5;241m=\u001b[39m train_data[idx_batch\u001b[38;5;241m*\u001b[39mbatch_size:(idx_batch\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m*\u001b[39mbatch_size,\u001b[38;5;241m-\u001b[39monehotsize:]\n\u001b[0;32m      9\u001b[0m mynetwork\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[1;32m---> 10\u001b[0m mynetwork\u001b[38;5;241m.\u001b[39mforward(batch_x)\n\u001b[0;32m     11\u001b[0m l\u001b[38;5;241m=\u001b[39mmynetwork\u001b[38;5;241m.\u001b[39mloss(batch_y)\n\u001b[0;32m     12\u001b[0m loss\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39ml\n",
      "File \u001b[1;32md:\\Users\\Fox\\桌面\\sad_machine_learning\\sad_network.py:22\u001b[0m, in \u001b[0;36mNeuralNetwork.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m,\u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m     21\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers:\n\u001b[1;32m---> 22\u001b[0m         \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m=\u001b[39mlayer\u001b[38;5;241m.\u001b[39mforward(\u001b[38;5;28minput\u001b[39m)\n\u001b[0;32m     23\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlossfunction\u001b[38;5;241m.\u001b[39mforward(\u001b[38;5;28minput\u001b[39m)\n",
      "File \u001b[1;32md:\\Users\\Fox\\桌面\\sad_machine_learning\\sad_layer.py:85\u001b[0m, in \u001b[0;36mFullConnectedLayer.forward\u001b[1;34m(self, input_data)\u001b[0m\n\u001b[0;32m     83\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m,input_data):\n\u001b[0;32m     84\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_data\u001b[38;5;241m=\u001b[39minput_data\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_size)\n\u001b[1;32m---> 85\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_data\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mmatmul(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_data,\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mw)\u001b[38;5;241m+\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mb\n\u001b[0;32m     86\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput_data\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "fc7=Network(\n",
    "    layers=[\n",
    "        FullConnectedLayer(784,256,std=0.001),\n",
    "        SigmoidLayer(),\n",
    "        FullConnectedLayer(256,64,std=0.001),\n",
    "        SigmoidLayer(),\n",
    "        FullConnectedLayer(64,10,std=0.001)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fc7,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:1762.213878597 \tAccuracy: 0.8821 \t\t\n",
      "epoch 2:\n",
      " Loss:703.103632425 \tAccuracy: 0.9049 \t\t\n",
      "epoch 3:\n",
      " Loss:588.020986274 \tAccuracy: 0.9169 \t\t\n",
      "epoch 4:\n",
      " Loss:522.313137608 \tAccuracy: 0.9273 \t\t\n",
      "epoch 5:\n",
      " Loss:473.416120541 \tAccuracy: 0.9286 \t\t\n",
      "epoch 6:\n",
      " Loss:435.184304176 \tAccuracy: 0.9376 \t\t\n",
      "epoch 7:\n",
      " Loss:403.558204384 \tAccuracy: 0.9395 \t\t\n",
      "epoch 8:\n",
      " Loss:377.574371840 \tAccuracy: 0.9446 \t\t\n",
      "epoch 9:\n",
      " Loss:355.244708458 \tAccuracy: 0.9472 \t\t\n",
      "epoch 10:\n",
      " Loss:336.148723354 \tAccuracy: 0.9488 \t\t\n"
     ]
    }
   ],
   "source": [
    "fcd_max=Network(\n",
    "    layers=[\n",
    "        MaxPooling2DLayer(2,2),\n",
    "        FullConnectedLayer(14*14,64),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(64,32),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(32,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fcd_max,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:1919.459553328 \tAccuracy: 0.8766 \t\t\n",
      "epoch 2:\n",
      " Loss:736.141086541 \tAccuracy: 0.9039 \t\t\n",
      "epoch 3:\n",
      " Loss:608.021984206 \tAccuracy: 0.9124 \t\t\n",
      "epoch 4:\n",
      " Loss:543.004604991 \tAccuracy: 0.9227 \t\t\n",
      "epoch 5:\n",
      " Loss:498.870583519 \tAccuracy: 0.929 \t\t\n",
      "epoch 6:\n",
      " Loss:463.383349653 \tAccuracy: 0.9338 \t\t\n",
      "epoch 7:\n",
      " Loss:434.667975763 \tAccuracy: 0.9389 \t\t\n",
      "epoch 8:\n",
      " Loss:408.327433644 \tAccuracy: 0.9413 \t\t\n",
      "epoch 9:\n",
      " Loss:385.163060428 \tAccuracy: 0.9449 \t\t\n",
      "epoch 10:\n",
      " Loss:364.167962744 \tAccuracy: 0.9465 \t\t\n"
     ]
    }
   ],
   "source": [
    "fcd_ave=Network(\n",
    "    layers=[\n",
    "        AveragePooling2DLayer(2,2),\n",
    "        FullConnectedLayer(14*14,64),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(64,32),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(32,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fcd_ave,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " Loss:966.007359795 \tAccuracy: 0.9334 \t\t\n",
      "epoch 2:\n",
      " Loss:381.252660092 \tAccuracy: 0.9491 \t\t\n",
      "epoch 3:\n",
      " Loss:289.024238364 \tAccuracy: 0.9589 \t\t\n",
      "epoch 4:\n",
      " Loss:240.555476085 \tAccuracy: 0.9626 \t\t\n",
      "epoch 5:\n",
      " Loss:207.102879801 \tAccuracy: 0.9686 \t\t\n",
      "epoch 6:\n",
      " Loss:182.223406861 \tAccuracy: 0.9691 \t\t\n",
      "epoch 7:\n",
      " Loss:164.912230295 \tAccuracy: 0.9683 \t\t\n",
      "epoch 8:\n",
      " Loss:149.418557357 \tAccuracy: 0.9729 \t\t\n",
      "epoch 9:\n",
      " Loss:136.055028022 \tAccuracy: 0.9714 \t\t\n",
      "epoch 10:\n",
      " Loss:124.627305235 \tAccuracy: 0.9688 \t\t\n"
     ]
    }
   ],
   "source": [
    "fcd_conv=Network(\n",
    "    layers=[\n",
    "        Conv2DLayer(1,3,3,0),\n",
    "        FullConnectedLayer(3*26*26,64),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(64,32),\n",
    "        ReLuLayer(),\n",
    "        FullConnectedLayer(32,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=fcd_conv,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1:\n",
      " loss:1.974804493 \tprocess: 30.45%\t   "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[30], line 16\u001b[0m\n\u001b[0;32m      1\u001b[0m conv2fc2\u001b[38;5;241m=\u001b[39mNetwork(\n\u001b[0;32m      2\u001b[0m     layers\u001b[38;5;241m=\u001b[39m[\n\u001b[0;32m      3\u001b[0m         Conv2DLayer(\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m16\u001b[39m,\u001b[38;5;241m3\u001b[39m,\u001b[38;5;241m0\u001b[39m),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     13\u001b[0m     lossfunction\u001b[38;5;241m=\u001b[39mSoftmaxCrossEntropyLossLayer()\n\u001b[0;32m     14\u001b[0m )\n\u001b[1;32m---> 16\u001b[0m train_and_test(\n\u001b[0;32m     17\u001b[0m     mynetwork\u001b[38;5;241m=\u001b[39mconv2fc2,\n\u001b[0;32m     18\u001b[0m     train_data\u001b[38;5;241m=\u001b[39mmnist_train_data,\n\u001b[0;32m     19\u001b[0m     test_data\u001b[38;5;241m=\u001b[39mmnist_test_data,\n\u001b[0;32m     20\u001b[0m     data_shape\u001b[38;5;241m=\u001b[39mmnist_shape,\n\u001b[0;32m     21\u001b[0m     onehotsize\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m,\n\u001b[0;32m     22\u001b[0m     batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m32\u001b[39m,\n\u001b[0;32m     23\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10\u001b[39m\n\u001b[0;32m     24\u001b[0m )\n",
      "Cell \u001b[1;32mIn[18], line 13\u001b[0m, in \u001b[0;36mtrain_and_test\u001b[1;34m(mynetwork, train_data, test_data, data_shape, onehotsize, batch_size, epoch_num, learning_rate, half_learning_rate_time)\u001b[0m\n\u001b[0;32m     11\u001b[0m l\u001b[38;5;241m=\u001b[39mmynetwork\u001b[38;5;241m.\u001b[39mloss(batch_y)\n\u001b[0;32m     12\u001b[0m loss\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39ml\n\u001b[1;32m---> 13\u001b[0m mynetwork\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[0;32m     14\u001b[0m mynetwork\u001b[38;5;241m.\u001b[39mupdate(learning_rate)\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m epoch_id\u001b[38;5;241m%\u001b[39mhalf_learning_rate_time\u001b[38;5;241m==\u001b[39m\u001b[38;5;241m0\u001b[39m: learning_rate\u001b[38;5;241m/\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m\n",
      "File \u001b[1;32md:\\Users\\Fox\\桌面\\sad_machine_learning\\sad_network.py:29\u001b[0m, in \u001b[0;36mNeuralNetwork.backward\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     27\u001b[0m grad\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlossfunction\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[0;32m     28\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mreversed\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlayers):\n\u001b[1;32m---> 29\u001b[0m     grad\u001b[38;5;241m=\u001b[39mlayer\u001b[38;5;241m.\u001b[39mbackward(grad)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "conv2fc2=Network(\n",
    "    layers=[\n",
    "        Conv2DLayer(1,16,3,0),\n",
    "        LeakyReLULayer(),\n",
    "        MaxPooling2DLayer(4,4),\n",
    "        Conv2DLayer(16,32,3,0),\n",
    "        LeakyReLULayer(),\n",
    "        MaxPooling2DLayer(4,4),\n",
    "        FullConnectedLayer(32,20),\n",
    "        LeakyReLULayer(),\n",
    "        FullConnectedLayer(20,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=conv2fc2,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv2fc1=Network(\n",
    "    layers=[\n",
    "        Conv2DLayer(1,10,3,0),\n",
    "        LeakyReLULayer(),\n",
    "        MaxPooling2DLayer(4,4),\n",
    "        Conv2DLayer(10,32,3,0),\n",
    "        LeakyReLULayer(),\n",
    "        MaxPooling2DLayer(4,4),\n",
    "        FullConnectedLayer(32,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=conv2fc1,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv2fc1_ave=Network(\n",
    "    layers=[\n",
    "        Conv2DLayer(1,10,3,0),\n",
    "        LeakyReLULayer(),\n",
    "        AveragePooling2DLayer(4,4),\n",
    "        Conv2DLayer(10,32,3,0),\n",
    "        LeakyReLULayer(),\n",
    "        MaxPooling2DLayer(4,4),\n",
    "        FullConnectedLayer(32,10)\n",
    "    ],\n",
    "    lossfunction=SoftmaxCrossEntropyLossLayer()\n",
    ")\n",
    "\n",
    "train_and_test(\n",
    "    mynetwork=conv2fc1_ave,\n",
    "    train_data=mnist_train_data,\n",
    "    test_data=mnist_test_data,\n",
    "    data_shape=mnist_shape,\n",
    "    onehotsize=10,\n",
    "    batch_size=32,\n",
    "    epoch_num=10\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "face",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
